import pandas as pd
import datetime
import DataProcess
import datetime
import math
import Core.Gadget as Gadget
import Core.DataSeries as DataSeries
import Core.WindFunctions as Wind


#---
def Download_MinuteBars(database, symbol, begDateTime, endDateTime):

    name = symbol + "_Time_60_Bar"
    dataObjects = DataSeries.DataSeries(name) # DataSeries

    fields = "open,high,low,close,volume,amt,oi";
    #param = "PriceAdj=B";
    param = ""
    data = w.wsi(symbol, fields, begDateTime, endDateTime, param);

    if data.ErrorCode != 0:
        Gadget.PlaySound()

    count = data.Data[0].__len__()
    for i in range(count):
        dataObject = {}
        datetime0 = data.Times[i]
        datetime0 = datetime0.replace(microsecond = 0)
        datetime1 =  datetime0 + timedelta(minutes=-1)
        datetime2 = datetime0 + timedelta(hours=-8) # 中国默认时区

        dataObject["DataSeries"] = name
        dataObject["DateTime"] = Gadget.ToDateTimeString(datetime0)
        dataObject["OpenDateTime"] = Gadget.ToDateTimeString(datetime1)
        dataObject["StdDateTime"] = datetime2
        dataObject["Key"] = dataObject["DataSeries"] + "_" + dataObject["DateTime"]

        dataObject["Symbol"] = symbol
        dataObject["Size"] = 60
        dataObject["BarType"] = "Time"

        dataObject["Open"] =  data.Data[0][i]
        dataObject["High"] = data.Data[1][i]
        dataObject["Low"] = data.Data[2][i]
        dataObject["Close"] = data.Data[3][i]
        dataObject["Volume"] = data.Data[4][i]
        dataObject["Money"] = data.Data[5][i]

        dataObject["OpenInt"] = 0
        value = data.Data[6][i]
        if not math.isnan(value):
            dataObject["OpenInt"] = value

        dataObjects.add(dataObject)
    #---把DataSeries储存到数据库---
    database.saveDataSeries(dataObjects, isUpSert=True)
    kkwood = 1


# ---Batch_Request_Write_Daily_Quote 的子函数---
def Download_DailyBars(database, symbol, instrument_type, begin_datetime, end_datetime, logger=None, save=True):

    time1 = datetime.datetime.now()
    #
    document_list = Wind.WSD_DailyQuote(symbol, instrument_type, begin_datetime, end_datetime, logger)
    #
    time2 = datetime.datetime.now()
    print("Request from Wind", begin_datetime, end_datetime, "Consume Time:" + str((time2 - time1).total_seconds()))

    # ---把DataSeries储存到数据库---
    if document_list == None or len(document_list) == 0:
        print(symbol + " No Data Requested")
        return
    #
    if save:
        database.Upsert_Many("financial_data", instrument_type + "_dailybar", {}, document_list)
        # database.SaveDataSeries(dataSeries, dataType="DailyBar", instrumentType=instrument_type)

    time3 = datetime.datetime.now()
    print("Save to DataBase Time:  " + str((time3 - time2).total_seconds()), "#Elements", len(document_list))
    return document_list


def Save_DaliyBars(database, symbol, instrument_type):
    pass


# ---use datetime1 & datetime2 to determin "Range of Calculate"
def DeterminRange(instrument, datetime1, datetime2):
    #
    # datetime1 = Gadget.ToUTCDateTime(datetime1)
    # datetime2 = Gadget.ToUTCDateTime(datetime2)
    #
    earlyReportDate = instrument["DateTime1"] # + timedelta(days=-365)
    realDateTime1 = datetime1  # 下限，最早的时间
    if earlyReportDate > datetime1:  # if 根本不需要取到这么早
        realDateTime1 = earlyReportDate

    realDateTime2 = datetime2  # 上限，最近的时间
    if instrument["DateTime2"] < datetime2:
        realDateTime2 = instrument["DateTime2"]

    #
    if realDateTime2 < realDateTime1:
        realDateTime1 = realDateTime2

    #
    realDateTime1 = Gadget.ToLocalDateTime(realDateTime1)
    realDateTime2 = Gadget.ToLocalDateTime(realDateTime2)
    return realDateTime1, realDateTime2


# --- Batch Request 大量获取，需要指定 起始日期 ---
# 先遍历 Instrument， 再遍历 dates
def Download_Data_ByDate(database, instruments, dates, callback, startIndex=0, logger=None, existedByDate={}):
    #
    count = 0
    # start = False
    # startSymbol = "300372.SZ"
    # start = True
    for instrument in instruments:
        symbol = instrument["symbol"]
        count = count + 1
        # if symbol == startSymbol:
        #    start = True
        # if start == False:
        #    continue
        if count < startIndex:
            continue

        print("")
        print("Download_Data " + symbol + " " + str(count))

        for reportDate in dates:
            # stdReportDate = Gadget.ToUTCDateTime(reportDate)
            #
            if instrument["datetime1"] > reportDate:
                continue
            #
            if instrument["datetime2"] < reportDate:
                continue
            #
            existedList = existedByDate.get(reportDate)
            if existedList and symbol in existedList:
                print(reportDate.date(), "Data Existed, Unnecessary to Update")
                continue
            #
            print(reportDate.date(), "Ready to Download", reportDate, " @ ", str(datetime.now()))
            callback(database, instrument, reportDate, logger)
            kkwood = 1
        # ---End of Lood Dates---
        kkwood = 2

# --- Batch Request 大量获取，需要指定 起始日期 ---
# 先遍历 dates， 再遍历 instruments
def Download_Data_ByDate2(database, instruments, dates, callback, startIndex=0, logger=None, existedByDate={}):
    # 自动去重机制
    cache = {}
    totalList = []
    for instrument in instruments:
        symbol = instrument["symbol"]
        cache[symbol] = instrument
        totalList.append(symbol)

    #
    df_instruments = Gadget.DocumentsToDataFrame(instruments)

    #
    for reportDate in dates:
        # 自动去重机制
        existedList = existedByDate.get(reportDate)
        if not existedList:
            existedList = []
        # existedList_t = np.array(existedList).reshape(-1, 1)
        # df_exist = pd.DataFrame(existedList_t, columns=["symbol"])
        # df_instruments.to_csv("d://list1.csv")
        # df_exist.to_csv("d://list2.csv")

        df_target = df_instruments[~df_instruments['symbol'].isin(existedList)]
        df_target = df_target[df_target["datetime1"] < reportDate]
        df_target = df_target[df_target["datetime2"] > reportDate]
        a = len(df_target)
        # targetList = list(set(totalList) - set(existedList))
        targetInstrument = df_target.to_dict(orient='records')
        count = 0
        for instrument in targetInstrument:
            symbol = instrument["symbol"]
            count = count + 1
            #
            if count < startIndex:
                continue
            #
            print("")
            print("Download_Data " + symbol + " " + str(count))
            #
            if symbol in existedList:
                print(reportDate.date(), "Data Existed, Unnecessary to Update")
                continue
            #
            if instrument["datetime1"] > reportDate:
                continue
            #
            if instrument["datetime2"] < reportDate:
                continue
            #
            print(reportDate.date(), "Ready to Download", reportDate, " @ ", str(datetime.now()))
            callback(database, instrument, reportDate, logger)
            kkwood = 1
        # ---End of Lood Dates---
        kkwood = 2


# --- 多个symbol 一次性Request ---
def Download_Data_MultSymbols(database, instruments, dates, callback):
    count = 0
    for reportDate in dates:
        # stdReportDate = Gadget.ToUTCDateTime(reportDate)
        stdReportDate = reportDate
        # print("Collect Data " + str(reportDate2))

        symbols = []
        for instrument in instruments:
            symbol = instrument["Symbol"]
            count = count + 1
            #
            if instrument["DateTime1"] > stdReportDate:
                continue
            #
            if instrument["DateTime2"] < stdReportDate:
                continue
            #
            symbols.append(symbol)
        # ---End of Loop Instruments---

        print("Collect Date: " + str(reportDate) + " @ " + str(datetime.now()))
        callback(database, symbols, reportDate)
        kkwood = 1

    kkwood = 2


#
def Download_Data(database, instruments, callback):
    count = 0
    for instrument in instruments:
        symbol = instrument["Symbol"]
        count = count + 1
        #
        print("Download_Data " + symbol + " " + str(count))
        #
        datetime1 = instrument["DateTime1"]
        datetime2 = instrument["DateTime2"]
        callback(database, instrument, datetime1, datetime2)
        kkwood = 1


#
def Download_Data2(database, instruments, datetime1, datetime2, callback):
    count = 0
    for instrument in instruments:
        symbol = instrument["Symbol"]
        count = count + 1
        #
        print("Download_Data " + symbol + " " + str(count))
        #
        realDateTime1, realDateTime2 = DeterminRange(instrument, datetime1, datetime2)
        callback(database, instrument, realDateTime1, realDateTime2)
        kkwood = 2


# ---Automatic 只需指定结束日期---
# ---批量获取 股票/期货、指数、债券、公募基金 日线---
def Automatic_Download_DailyQuote(database, end_date, instrument_type="Stock", outside_instruments=None, start_index=0,
                                  config=None, too_early_date=None):
    #
    if config != None:
        logger = config.Logger("Download_DailyQuote")
    else:
        logger = None

    sort = [("DateTime", 1)]
    datetime1_field = "datetime1"
    datetime2_field = "datetime2"

    instruments = []
    if outside_instruments != None:
        for symbol in outside_instruments:
            rets = database.Find("financial_data", "instrument_" + instrument_type, filter={"Symbol": symbol})
            if len(rets) != 0:
                instruments.append(rets[0])
    elif instrument_type == "Future":
        instruments = database.Find("financial_data", "instrument_" + instrument_type)
        instruments_future_index = database.Find("financial_data", "FutureIndex")
        instruments = instruments + instruments_future_index
    elif instrument_type == "Option":
        instruments = database.Find("financial_data", "instrument_" + instrument_type)
    elif instrument_type == "MutualFund":
        x_filter = []
        x_filter = [("Invest_Type1", "混合型基金"), ("or", "Invest_Type1", "股票型基金")]
        # x_filter = [("Invest_Type1", "债券型基金")]
        # x_filter = [("Invest_Type1", "货币市场型基金")]
        # x_filter = [("Invest_Type1", "国际(QDII)基金"), ("or", "Invest_Type1", "另类投资基金")]
        instruments = database.Find("financial_data", "instrument_mutualfund", x_filter)
    else:
        instruments = database.Find("financial_data", "instrument_" + instrument_type)

    # --- used for Options---
    if instrument_type == "Option":
        undelySymbols= []
        for instrument in instruments:
            if "Underly" in instrument:
                underly = instrument["Underly"]
                if underly not in undelySymbols:
                    undelySymbols.append(underly)

        # ---Take underly before instruments---
        undelyInstruments = []
        if len(undelySymbols) > 0:
            for symbol in undelySymbols:
                instrument = None
                tempSymbol = symbol.split(".")
                underly = database.Find("financial_data", "instrument_MutualFund", filter={"Symbol": tempSymbol[0] + ".OF"})
                if len(underly) != 0: # is ETFOption
                    instrument = underly[0]
                    instrument["Symbol"] = symbol
                    instrument["InstrumentType"] = "Stock"
                else:
                    if len(underly) == 0:
                        underly = database.Find("financial_data", "instruments_Stock", filter={"Symbol":symbol})
                    if len(underly) == 0:
                        underly = database.Find("financial_data", "instruments_Future", filter={"Symbol": symbol})
                    if len(underly) == 0:
                        print("Can't find Underly " + symbol)
                    else:
                        instrument = underly[0]

                if instrument:
                    undelyInstruments.append(instrument)

            undelyInstruments.extend(instruments)
            instruments = undelyInstruments

    start = False
    count = 0
    end_date = Gadget.ToDate(end_date)
    for instrument in instruments:
        count += 1
        #
        symbol = instrument.get("symbol")
        if not symbol:
            symbol = instrument.get("Symbol")

        if count < start_index:
            continue
        #
        # if symbol != "H11001.CSI":
        #     continue
        #
        # if symbol != "SPX.GI":
        #     continue

        # 校正时间
        begin_download_date = instrument[datetime1_field]
        date1 = Gadget.ToDate(begin_download_date)
        #
        end_download_date = instrument[datetime2_field]
        date2 = Gadget.ToDate(end_download_date)

        # 如果指定了tooEarlyDate, 那么不再更新太古老的Instrument
        if too_early_date and end_download_date < too_early_date:
            continue

        # Invalid DateTime1, 可能该券，合约尚未上市
        if date1 < datetime.datetime(1900, 1, 1):
            print(symbol, "Invalid DateTime1, may not on List")
            continue

        #
        print("Download_DailyQuote", symbol, "Count", count)
        #
        instrument_type = instrument["instrument_type"]
        if instrument_type == "FutureIndex":
            instrument_type = "Future"
        collectionName = symbol + "_Time_86400_Bar"

        # ---Check dataSeries in Database---
        load_begin_time = datetime.datetime.now()

        # ---读取数据局，确定队尾时间---
        df = database.Get_Daily_Bar_DataFrame(symbol, instrument_type=instrument_type)
        length = len(df)
        load_finish_time = datetime.datetime.now()
        print("Load Database Time: " + str((load_finish_time - load_begin_time).total_seconds()))

        # ---确定下载数据开始时间：DateTime1---
        if length == 0:  # 数据不存在, 首次插入数据---
            begin_download_date = date1
            print("First Build Daily Quote: " + collectionName)

        else:  # 数据存在，确认上次队尾数据
            # DT1
            begin_download_date = df.iloc[-1]["date"]

            # 数据最后一天顺延一天
            begin_download_date = begin_download_date + datetime.timedelta(days=1)

        # ---确认结束下载数据时间：DateTime2---
        # Check if Delisted, 检查是否退市，到期
        is_delisted = False
        if date2 < end_date:
            is_delisted = True
        #
        if is_delisted:  # 已经下市
            end_download_date = date2
        else:  # 仍在交易
            end_download_date = end_date

        # ---特殊情况处理---
        if begin_download_date > end_download_date:
            print(symbol, "Dt1 ", begin_download_date, " > Dt2", end_download_date, "Unnassary to Request")
            continue

        # ---Download Daily Bars---
        DataProcess.General.Download_DailyBars(database, symbol, instrument_type, begin_download_date, end_download_date, logger)

        time4 = datetime.datetime.now()
        print("Total time:  ", str((time4 - load_begin_time).total_seconds()))

        #---Option Bar 需要额外信息计算---
        #if instrumentType == "Option":
        #    OptionGadget.CalcDailyBarInfo()
        kkwood = 0


# ---cold start---
def Download_Daily_Quote(database, datetime1=None, datetime2=None, instrument_type="Stock", start_index=0):
    #
    instruments = database.Find("Instruments", instrument_type)
    #
    count = 0
    for instrument in instruments:
        count += 1
        #
        symbol = instrument.get("symbol")
        #
        if count < start_index:
            continue
        #
        real_datetime1 = datetime1
        if datetime1 == None:
            real_datetime1 = instrument.get("datetime1")
        #
        real_datetime2 = datetime2
        if datetime2 == None:
            real_datetime2 = instrument.get("datetime2")

        # ---Download Daily Bars---
        print("Downloading", count, symbol, real_datetime1.date(), "To", real_datetime2.date(), "Update Time", datetime.datetime.now())
        if symbol == "10001734.SH":
            a = 0
        DataProcess.Download_DailyBars(database, symbol, instrument_type, real_datetime1, real_datetime2, None)
        a = 0


# ---批量获取 财务数据---
def Automatic_Download_Fundamentals(database, endDate, startIndex=0):
    print("Batch Request Financial Fundameantals Starts ...")
    datetime1 = datetime(2005, 1, 2)
    year = endDate.year
    lastYear = endDate.year - 1
    datetime2 = datetime(lastYear, 9, 30)
    reportDateTimes = Gadget.GenerateReportDates(datetime1, datetime2)

    # --- 自动按照当前日期，决定是否补齐---
    if endDate.month >= 2:
        reportDateTimes.append(datetime(lastYear,12,31))
        reportDateTimes.append(datetime(year, 3, 31))
        datetime2 = datetime(year, 3, 31)
    if endDate.month >= 7:
        reportDateTimes.append(datetime(year, 6, 30))
        datetime2 = datetime(year, 6, 30)
    if endDate.month >= 10:
        reportDateTimes.append(datetime(year, 9, 30))
        datetime2 = datetime(year, 9, 30)

    #
    Batch_Download_Fundamentals_Adv(database, datetime1, datetime2, startIndex=startIndex)
    Check_Fundamental_DownloadProgress(database, datetime2)


#---批量获取 历史行业分类---
def Automatic_Download_HistoricalIndustry(database):
    pass


def Automatic_Download_Instruments(database, datetime2):
    Request_Write_Index_List(database, datetime2)
    Download_Stock_List(database, datetime2)
    Request_Write_Bond_List(database, datetime2)
    #Request_Write_MutualFund_List(database, datetime2)


# 通过 report date 判断已经存在的数据，不必重新Request
# 如财务报表，财务快报数据
def GetExsited(database, databaseName, tableName, reportDates):
    existedByDate = {}
    #
    for reportDate in reportDates:
        existed = database.Find(databaseName, tableName, filter={"Report_Date": reportDate.date()},
                                projection=["symbol"])
        existedList = []
        for e in existed:
            existedList.append(e["symbol"])
        existedByDate[reportDate] = existedList
    #
    return existedByDate


# 自动判断缺失报表的日期
# 用于更新 Fundamental / MutualFund
def GetMissingReportDate(database, databaseName, tableName, instruments, datetime1, datetime2):
    # 确定目标数据表中已存在的Report Dates
    query = {}
    query.append(("report_date", ">=", datetime1))
    query.append(("report_date", "<=", datetime2))
    documents = database.Find(databaseName, tableName, filter=query, projection=["symbol", "report_date"])
    df = Gadget.DocumentsToDataFrame(documents)
    #
    reportDatesBySymbol = {}
    dfGroups = df.groupby("symbol")
    for dfGroup in dfGroups:
        symbol = dfGroup[0]
        reportDatesBySymbol[symbol] = list(dfGroup[1])

    #
    targetReportDateBySymbol = {}
    for instrument in instruments:
        if instrument["datetime1"] < datetime(1900, 1, 1):
            continue
        # 未上市
        if instrument["datetime1"] > datetime2:
            continue
        # 已下市
        if instrument["datetime2"] < datetime1:
            continue
        #
        realDateTime1 = max(instrument["datetime1"], datetime1)
        realDateTime2 = min(instrument["datetime2"], datetime2)
        reportDatesTotal = Gadget.GenerateReportDates(realDateTime1, realDateTime2, asDate=True)
        #
        symbol = instrument["symbol"]
        if symbol in reportDatesBySymbol:
            reportDatesInDataBase = reportDatesBySymbol.get(symbol)
        else:
            reportDatesInDataBase = []
        #
        reportDates = list(set(reportDatesTotal).difference(set(reportDatesInDataBase)))
        targetReportDateBySymbol[symbol] = reportDates
    #
    return targetReportDateBySymbol


# 检查Bar完整性，考虑合并
def CheckMissingBar(instrument, bmBars, bars):
    barByDate = {}
    missingDateTime = []
    #
    for bar in bars:
        barByDate[bar["StdDateTime"]] = bar

    for bmBar in bmBars:
        stdDateTime = bmBar["StdDateTime"]
        if barByDate.get(stdDateTime) == None:
            missingDateTime.append(stdDateTime)

    for stdDateTime in missingDateTime:
        datetime1 = Gadget.ToLocalDateTime(stdDateTime)
        datetime1 = Gadget.ToDate(datetime1)
        # ---Download Daily Bars---
        DataProcess.Download_DailyBars(database, instrument, datetime1, datetime1)


# 检查Bar完整性，考虑合并
def CheckUnCompleteBar():
    datetime1 = datetime.datetime(2017,5,24)
    datetime1 = Gadget.ToUTCDateTime(datetime1)
    #
    datetime2 = datetime.datetime(2017,7,26)
    datetime2 = Gadget.ToUTCDateTime(datetime2)
    #
    Wind.w.start()
    #Wind.Download_DailyBars(database, "000300.SH", datetime1, datetime2, "Index")
    #Wind.Download_DailyBars(database, "10000905.SH", datetime1, datetime2, "Option")

    #
    options = database.find("Instruments", "Option", query={"Code": "510050.SH"})
    for option in options:
        symbol = option["Symbol"]
        stdDateTiem1 = option["DateTime1"]
        datetime1 = Gadget.ToLocalDateTime(stdDateTiem1)
        stdDateTiem2 = option["DateTime2"]
        stdDateTiem2 += datetime.timedelta(hours=15)
        datetime2 = Gadget.ToLocalDateTime(stdDateTiem2)
        #
        bmBars = database.find("Index", "000300.SH_Time_86400_Bar", datetime1, datetime2)
        #
        bars = database.find("Option", symbol + "_Time_86400_Bar")
        if len(bars) == 0:
            print("Missing Total Data " + symbol)
            Wind.Request_Write_DailyBars(database, symbol, datetime1, datetime2, "Option")
            continue
        #
        if len(bars) != (len(bmBars)):
            print("Missing Data " + symbol + " #Bar " + str(len(bars)) + " #BMBars " + str(len(bmBars)))
            database.DeleteCollections("Option", pattern=symbol)
            Wind.Request_Write_DailyBars(database, symbol, datetime1, datetime2, "Option")
            continue